package sis.ra.partofspeech;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeSet;

import sis.ra.utility.InxEntity;
import sis.ra.utility.Utils;


public class POSandNE {
	
	public static HashMap<Integer, wordindex> pos (String text, String htformat)
	{
		String linguistxpath="c:\\Program Files\\INXIGHT\\LinguistxPlatform3.7\\lx-3\\";
		String posresult="";
		Utils.writeFile(linguistxpath+"windows-2003-i686-msvc-7.1-32\\test.txt ", text);
		String s="";
		String encoding="";
		String format="";
		String language="";
		String sentencestart="";
		String sentenceend="";
		int start=0;
		int end=0;
		int offset=0;
		Integer count=0;
		HashMap<Integer,wordindex> sentenceindex=new HashMap<Integer, wordindex> ();
		ArrayList<wordindex> sentenceindexArray=new ArrayList<wordindex> ();
		ArrayList<wordindex> posindexArray=new ArrayList<wordindex> ();
		
		String cmd="\""+linguistxpath+"windows-2003-i686-msvc-7.1-32\\test_platform.exe\" -k 735442535d524e "+
//		" -tag " +
		" -tokenize " + 
		"-d " + "\""+linguistxpath + "lang\"" + 
//		" -stem " +
		" -"+htformat+" " +"\"" +linguistxpath + "windows-2003-i686-msvc-7.1-32\\test.txt\" ";
		try {
//			System.out.println(cmd);
		     Process p = Runtime.getRuntime().exec(cmd);
		    
		    BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
		    BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream()));
		    while ((s = stdInput.readLine()) != null)   
		    {
//		    	System.out.println(s);
//		    	if (s.contains("The character encoding of this segment is")) 
//		    	{
//		    		encoding=s.substring(s.indexOf(":")+1).trim(); 
//		    	} else
//		    	if (s.contains("The format of this segment is")) 
//		    	{
//		    		format=s.substring(s.indexOf(":")+1).trim(); 
//		    	}else
//		    	if (s.contains("The languages (and their respective offsets) of this segment are"))  
//		    	{
//		    		s=stdInput.readLine();
//		    		language=s.substring(0,s.indexOf("start")).trim();	    	
//		    	} else
//		    		{
//		    			
//		    		}
//	    		}	
		    	
		    	// analyze token part
		    	if (s.contains("segment tokenization -- tokens:"))
		    	{
		    		while (!(s= stdInput.readLine()).contains("segment tokenization -- normalized tokens:"))
		    		{
		    			if (s.startsWith("  ")&&s.contains("start=")&&s.contains("end=")&&!s.startsWith("     "))
		    				{
		//    					System.out.println(s);
		    					String [] t=s.split("\t");
		    					if (t.length<4) continue;
		    					//wordindex(String pos, Integer position, String token, String type, String word)
		    					wordindex wi=new wordindex("", Integer.parseInt(t[2].substring(6).trim()),"","", t[0].trim());
		    					sentenceindex.put(Integer.parseInt(t[2].substring(6).trim()), wi);
		    					sentenceindexArray.add(wi);
		    				}
		    		}
		    	}
		    	
		    	if (s.contains("segment tokenization -- tags:"))
		    	{
		    		
		    		while ((s= stdInput.readLine())!=null&&!s.contains("The character encoding of this segment is"))
		    		{
		    			
		    			if (s.startsWith("  ")&&s.trim().length()>2)
		    			{
		    				System.out.println("=="+s);
		    				String [] t=s.split("\t");
		    				//public wordindex(String entity, String entitytype, String pos,Integer position, String token, String type, String word) {
//		    				System.out.println(count+" "+sentenceindexArray.get(count).getWord());
		    				sentenceindexArray.get(count).setPos(t[1].trim());
		    				
		    				count=count+1;
		    			}		    			
		    		}
		    	}
		    }
	//	    while ((s = stdError.readLine()) != null)   System.out.println(s);
		}
		catch (IOException e) {
			    e.printStackTrace();    
			}

		return sentenceindex;
	}
	
	public static HashMap<Integer, wordindex> posNeAlias(String contents)
	{
		/*
		 * generate entity, put to HashMap<Integer,InxEntity> nes
		 */
		String ne=Utils.getNE(contents);

//		Utils.writeFile("C:\\Documents and Settings\\I820753\\Desktop\\Madonna\\wiki_test_ne.txt", ne);
		HashMap<Integer,InxEntity> nes=Utils.parseIMSXMLContentHash(ne);
		HashMap<Integer,String> aliasindex=AlliasAnalysis.aliasAnalysis(ne);
//		System.out.println("size"+aliasindex.size());
//		TreeSet<Integer> sortedSetpos=new TreeSet<Integer>(aliasindex.keySet());
//		Object [] sortedArrayne=sortedSetpos.toArray();
	
//		for (int i=0;i<sortedArrayne.length;i++)
//		{
//			System.out.println(sortedArrayne[i]+" "+aliasindex.get(sortedArrayne[i]));
//		}
		/*
		 * sort the entities by document offset
		 */
		TreeSet<Integer> sortedSetne=new TreeSet<Integer>(nes.keySet());
		Object [] sortedArrayne=sortedSetne.toArray();
	
//		for (int i=0;i<sortedArrayne.length;i++)
//		{
//			System.out.println(sortedArrayne[i]+" "+nes.get(sortedArrayne[i]).getSurface()+" "+nes.get(sortedArrayne[i]).getEntityType());
//			if (aliasindex.containsKey(sortedArrayne[i]))
//				System.out.println(aliasindex.get(sortedArrayne[i]));
//		}
//		
		/*
		 * part of speech, put into HashMap<Integer,wordindex>
		 */
		HashMap<Integer,wordindex> posresult=pos(contents, "text");

		/*
		 * sort the pos results
		 */
		TreeSet<Integer> sortedSet=new TreeSet<Integer>(posresult.keySet());
		Object [] sortedArray=sortedSet.toArray();
	
		/*
		 * add entities to wordindex
		 */
		for (int i=0;i<sortedArrayne.length;i++)
		{
			Integer neindexnow=(Integer)sortedArrayne[i];
			Integer posindexnow=neindexnow;
//			System.out.println(nes.get(neindexnow).getSurface()+" "+nes.get(neindexnow).getEntityType());
			if (posresult.containsKey(posindexnow)&&nes.get(neindexnow).getSurface().startsWith(posresult.get(posindexnow).getWord()) ) 
			{
				
//				System.out.println(neindexnow+" "+nes.get(neindexnow).getEntityType()+" "+nes.get(neindexnow).getSurface());
				posresult.get(posindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
				posresult.get(posindexnow).setEntity(nes.get(neindexnow).getSurface());	
				if (aliasindex.containsKey(neindexnow))
					posresult.get(posindexnow).setReference(aliasindex.get(neindexnow).split("\t")[1]);
				//System.out.println(neindexnow+" "+nes.get(neindexnow).getEntityType()+" "+nes.get(neindexnow).getSurface());
			}else
			{
				posindexnow=posindexnow-10;
				int loop=0;
				while (!posresult.containsKey(posindexnow)||!nes.get(neindexnow).getSurface().toLowerCase().trim().contains(posresult.get(posindexnow).getWord().toLowerCase().trim()))
				{
					posindexnow=posindexnow+1;
//					System.out.println("=="+posindexnow+" "+nes.get(neindexnow).getSurface().toLowerCase().trim());
					loop=loop+1;
					if (loop>20) break;					
				}
				if (loop>20) 
				{
					neindexnow=(Integer)sortedArrayne[i];
//					posresult.get(neindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
//					posresult.get(neindexnow).setEntity(nes.get(neindexnow).getSurface());		
				}else{
				posresult.get(posindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
				posresult.get(posindexnow).setEntity(nes.get(neindexnow).getSurface());	
				if (aliasindex.containsKey(neindexnow))
					posresult.get(posindexnow).setReference(aliasindex.get(neindexnow).split("\t")[1]);
				}
			}
		}
		return posresult;
	}
	
	private static HashMap<Integer, String> aliasAnalysis(String ne) {
		// TODO Auto-generated method stub
		return null;
	}

	public static String toString(HashMap<Integer,wordindex> posresult)
	{
		
		TreeSet<Integer> sortedSet=new TreeSet<Integer>(posresult.keySet());
		Object [] sortedArray=sortedSet.toArray();

		//output the wordindexresult
		String out="";
		for (int i=0;i<sortedArray.length;i++)
		{
			if (posresult.get(sortedArray[i]).getEntity()==null)
				out=out+sortedArray[i]+"\t"+posresult.get(sortedArray[i]).getWord()+"\t"+posresult.get(sortedArray[i]).getPos()+"\n";
			else
			{
				if (posresult.get(sortedArray[i]).getReference()==null)
				out=out+sortedArray[i]+"\t"+posresult.get(sortedArray[i]).getWord()+"\t"+posresult.get(sortedArray[i]).getPos()+"\t"+
					posresult.get(sortedArray[i]).getEntity()+"\t"+posresult.get(sortedArray[i]).getEntitytype()+"\n";		
				else
					out=out+sortedArray[i]+"\t"+posresult.get(sortedArray[i]).getWord()+"\t"+posresult.get(sortedArray[i]).getPos()+"\t"+
					posresult.get(sortedArray[i]).getEntity()+"\t"+posresult.get(sortedArray[i]).getEntitytype()+"\t"+posresult.get(sortedArray[i]).getReference()+"\n";		
			}
		}
		return out;
	}
	public static void main(String[] args) {
		
//		String url="http://www.cnn.com/2008/SHOWBIZ/Music/08/15/mtv.vma.nominees.ap/index.html?iref=newssearch";
//		//"http://www.cnn.com/2008/SHOWBIZ/08/20/broadway.woes.ap/index.html?iref=newssearch";
//		//"http://cnn.com/2001/TECH/industry/06/14/sap.palm.crm.idg/index.html?iref=newssearch";
//		//"http://archives.cnn.com/2000/TECH/computing/01/20/mysap.linux.idg/index.html";
//		//"http://www.cnn.com/2007/WORLD/asiapcf/10/04/koreas.summit/index.html";
//		//"http://www.cnn.com/2007/WORLD/africa/10/04/congo.crash.ap/index.html";
		
		/*
		 * crawling clean data from CNN
		 */
//		String contents=Utils.fetchContentfrURL(url);
////		contents=Utils.getPagePlainContent(url);
////		System.out.println(contents);
//		String news=cnn.cnnOnePage(contents);
//	
//		System.out.println(news);

//		String news=Utils.readFile("C:\\Documents and Settings\\I820753\\Desktop\\Madonna\\wiki_test.txt");
		
//		String test= "Between 2am and 4am on a Monday morning, six new unregistered BMW vehicles were stolen from a car dealership in a western suburb of Bluesky.  The stolen cars were held in a locked storage garage toward the rear of the dealership. The vehicles were tightly parked at the back of the building.  The keys were taken from a lockbox in the office.  A window was broken in the office and was used to gain access to the office area.  The dealership where the thefts occurred is of substantial size and although it fronts on to a main road, it backs on to an industrial estate.  The dealerships security patrol operates only at the front of the business from the main road.  The thieves accessed the dealership by cutting through a wire fence at the rear of the premises.  They then removed the tin sheeting to enter the storage garage area.  This shows knowledge of the premises.  This would either indicate an inside job or that one, or more, of the thieves might have conducted reconnaissance visits to the site during business hours. ";
		String pathname="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\";
		File path=new File(pathname);
		File[] names=path.listFiles();
//		for (int i=0;i<names.length;i++)
//		{
//			String filename=names[i].toString();
//			if (!filename.contains("wiki")) continue;
		String filename = "C:\\Documents and Settings\\I820753\\Desktop\\dataset\\AT&T_wiki.txt";
			System.out.println(filename);
		String news=Utils.readFile(filename);
		HashMap<Integer,wordindex> posresult=posNeAlias(news);
//		/*
//		 * generate entity, put to HashMap<Integer,InxEntity> nes
//		 */
//		String ne=Utils.getNE(news);
//
////		Utils.writeFile("C:\\Documents and Settings\\I820753\\Desktop\\Madonna\\wiki_test_ne.txt", ne);
//		HashMap<Integer,InxEntity> nes=Utils.parseIMSXMLContentHash(ne);
//		
//		/*
//		 * sort the entities by document offset
//		 */
//		TreeSet<Integer> sortedSetne=new TreeSet<Integer>(nes.keySet());
//		Object [] sortedArrayne=sortedSetne.toArray();
//	
////		for (int i=0;i<sortedArrayne.length;i++)
////		{
////			System.out.println(sortedArrayne[i]+" "+nes.get(sortedArrayne[i]).getSurface()+" "+nes.get(sortedArrayne[i]).getEntityType());
////		}
//		
//		/*
//		 * part of speech, put into HashMap<Integer,wordindex>
//		 */
//		HashMap<Integer,wordindex> posresult=pos(news, "text");
//
//		/*
//		 * sort the pos results
//		 */
//		TreeSet<Integer> sortedSet=new TreeSet<Integer>(posresult.keySet());
//		Object [] sortedArray=sortedSet.toArray();
//	
//		/*
//		 * add entities to wordindex
//		 */
//		for (int i=0;i<sortedArrayne.length;i++)
//		{
//			Integer neindexnow=(Integer)sortedArrayne[i];
//			Integer posindexnow=neindexnow;
//		//	System.out.println(nes.get(neindexnow).getSurface()+" "+nes.get(neindexnow).getEntityType());
//			if (posresult.containsKey(posindexnow)&&nes.get(neindexnow).getSurface().startsWith(posresult.get(posindexnow).getWord()) ) 
//			{
//				
////				System.out.println(neindexnow+" "+nes.get(neindexnow).getEntityType()+" "+nes.get(neindexnow).getSurface());
//				posresult.get(posindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
//				posresult.get(posindexnow).setEntity(nes.get(neindexnow).getSurface());		
//				//System.out.println(neindexnow+" "+nes.get(neindexnow).getEntityType()+" "+nes.get(neindexnow).getSurface());
//			}else
//			{
//				posindexnow=posindexnow-10;
//				int loop=0;
//				while (!posresult.containsKey(posindexnow)||!nes.get(neindexnow).getSurface().toLowerCase().trim().contains(posresult.get(posindexnow).getWord().toLowerCase().trim()))
//				{
//					posindexnow=posindexnow+1;
//					System.out.println("=="+posindexnow+" "+nes.get(neindexnow).getSurface().toLowerCase().trim());
//					loop=loop+1;
//					if (loop>20) break;					
//				}
//				posresult.get(posindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
//				posresult.get(posindexnow).setEntity(nes.get(neindexnow).getSurface());		
//				}
//		}
	
		String out=POSandNE.toString(posresult);
		
		System.out.println(out);
		Utils.writeFile(filename.replace("wiki", "analysis"), out);
//		Utils.writeFile("C:\\Documents and Settings\\I820753\\Desktop\\Madonna\\wiki_test_en_pos.txt", out);
		}
//	}
}